# -*- coding: utf-8 -*-
"""
Created on 2021-12-15 21:27:28
---------
@summary:
---------
@author: 74793
"""

import feapder
from items import job_detail_item, job_menu_item


class IndexSpider(feapder.AirSpider):
    def start_requests(self):
        yield feapder.Request("https://www.lagou.com/")

    def parse(self, request, response):
        mainMenu = response.xpath('//div[@class="menu_box"]//div[@class="menu_sub dn"]')

        print(len(mainMenu))

        dic_detail_job = {}

        for menu in mainMenu:
            # title表示行业类别
            title = menu.xpath('./p/text()').extract_first()
            # jobs表示行业里具体分类有哪些
            jobs = menu.xpath('./dl')

            for job in jobs:
                sub_title = job.xpath('./dt/span/text()').extract_first()
                # detail_job具体的工作类别
                detail_job = job.xpath('./dd/a/h3/text()').extract()

                dic_detail_job[sub_title] = detail_job

                jobMenu = job_menu_item.JobMenuItem()
                jobMenu.title = title
                jobMenu.sub_title = sub_title

                yield jobMenu
                # print(title, sub_title, detail_job, sep="\033[31m|\033[0m")

        # print(dic_detail_job)
        for key, values in dic_detail_job.items():
            for value in values:
                jobDetail = job_detail_item.JobDetailItem()
                jobDetail.sub_title = key
                jobDetail.detail_job = value

                # print(key, value)

                yield jobDetail



if __name__ == "__main__":
    IndexSpider().start()